#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from scrapy.selector import Selector
from datetime import datetime
import os
import csv
import re
def getNames():
    excel = open('./config/pandaName.txt', 'r', encoding='UTF-8').read()
    data = {}
    rows = excel.splitlines()
    for row in rows:
        fields = row.split('\t')
        data[fields[1]] = fields
    return data
print('start:', datetime.now())
names = getNames()
f = open('./data/pandalist.csv', 'w', encoding='utf-8-sig')
fieldnames = ['id', 'name', 'sex', 'family', 'story']
rows = [['id', 'name', 'sex', 'family', 'story']]
writer = csv.writer(f)
for filename in os.listdir('./data/pandapia/'):
    if(os.path.splitext(filename)[1]!='.html'):
        continue
    pid = os.path.basename(filename).split('.')[0]
    html = open('./data/pandapia/'+filename, 'r', encoding='UTF-8').read()
    s = Selector(text=html)
    name = s.xpath('string(//div[@class="panda_font_name"]//span[1]//text())').extract()
    if(names.get(name[0]) == None):
        continue
    divs = s.xpath('//div[@class="timel"]')
    story = ''
    family = ''
    for div in divs:
        title = div.xpath('.//h4//text()').extract()[0]
        if(title == '家族'):
            for li in div.xpath('.//div[@class="time_family_fl"]//li'):
                link = li.xpath('.//a[contains(@href, "javascript")]/@onclick').extract()
                linkinfo = re.match('.*?id=(.*?)\'$', link[0])
                family = family+li.xpath('.//h4//text()').extract()[0]+'|'+li.xpath('.//p//text()').extract()[0]+'|'+linkinfo.group(1)+'\n'
        else:
            story = story+title+'\n'+div.xpath('.//p//text()').extract()[0]+'\n'
    rows.append([pid, name[0], names[name[0]][2], family, story])
    #rows.append({'id':pid, 'name':name[0], 'sex':names[name[0]][2], 'family':family, 'story':story})
writer.writerows(rows)
f.close()
print('end:', datetime.now())